# -*- coding: utf-8 -*-
"""
文件: tmdb_data_preprocessing.py
功能: 对 tmdb_5000_credits.csv 进行完整分析与预处理
"""

import pandas as pd
import json
import ast
import warnings
import chardet  # 用于检测文件编码

# 忽略警告
warnings.filterwarnings('ignore')

print("🚀 开始处理 TMDB 5000 Credits 数据集...\n")

# ========================
# 1. 加载原始数据（自动检测编码）
# ========================
try:
    # 检测文件编码
    with open('tmdb_5000_credits.csv', 'rb') as f:
        result = chardet.detect(f.read(10000))
        detected_encoding = result['encoding']
        print(f"🔍 检测到文件编码: {detected_encoding}")

    # 使用检测到的编码读取 CSV
    df = pd.read_csv('tmdb_5000_credits.csv', encoding=detected_encoding)
except FileNotFoundError:
    raise FileNotFoundError("请确保 'tmdb_5000_credits.csv' 文件在当前目录下！")
except Exception as e:
    print(f"❌ 读取文件失败，尝试使用 latin1 编码...")
    try:
        df = pd.read_csv('tmdb_5000_credits.csv', encoding='latin1')
    except Exception as e2:
        raise RuntimeError(f"无法读取文件，请检查文件路径和格式：{e2}")

print(f"✅ 原始数据形状: {df.shape}")
print(f"📊 列名: {list(df.columns)}\n")

# 检查前几行
print("🔍 原始数据前两行预览:")
print(df.head(2), "\n")

# 检查缺失值
print("🧩 缺失值统计:")
print(df.isnull().sum(), "\n")


# ========================
# 2. 安全解析 JSON 字段（cast 和 crew）
# ========================
def safe_parse_json(x):
    """
    安全解析 JSON 或类似 JSON 的字符串
    使用 ast.literal_eval 更容错，避免因单引号、非标准格式导致失败
    """
    if pd.isna(x) or x.strip() == '':
        return []
    try:
        return ast.literal_eval(x)  # 比 json.loads 更强大
    except Exception as e:
        # 尝试修复常见问题（如单引号）
        try:
            x_fixed = x.replace("'", '"').replace('None', 'null').replace('True', 'true').replace('False', 'false')
            return json.loads(x_fixed)
        except:
            print(f"⚠️ 无法解析字段: {x[:50]}...")
            return []


print("🔧 正在解析 'cast' 和 'crew' 字段...")

df['cast'] = df['cast'].astype(str).apply(safe_parse_json)
df['crew'] = df['crew'].astype(str).apply(safe_parse_json)

# 验证解析结果
cast_lengths = df['cast'].apply(len)
crew_lengths = df['crew'].apply(len)

print(f"✅ cast 字段平均人数: {cast_lengths.mean():.1f}")
print(f"✅ crew 字段平均人数: {crew_lengths.mean():.1f}")
print(f"📈 cast 最大人数: {cast_lengths.max()}, crew 最大人数: {crew_lengths.max()}\n")


# ========================
# 3. 提取关键特征
# ========================

print("🎯 正在提取关键特征...\n")

# (1) 提取前3位主演姓名
df['top_3_actors'] = df['cast'].apply(
    lambda x: [actor['name'] for actor in x[:3] if 'name' in actor] if isinstance(x, list) else []
)

# (2) 提取导演（Director）
df['director'] = df['crew'].apply(
    lambda x: next((member['name'] for member in x if isinstance(member, dict) and member.get('job') == 'Director'), None)
    if isinstance(x, list) else None
)

# (3) 提取编剧（Writer）
df['writers'] = df['crew'].apply(
    lambda x: [member['name'] for member in x if isinstance(member, dict) and member.get('department') == 'Writing']
    if isinstance(x, list) else []
)

# (4) 提取摄影指导（Director of Photography）
df['cinematographer'] = df['crew'].apply(
    lambda x: next((member['name'] for member in x if isinstance(member, dict) and member.get('job') == 'Director of Photography'), None)
    if isinstance(x, list) else None
)

# (5) 统计团队规模
df['cast_size'] = df['cast'].apply(lambda x: len(x) if isinstance(x, list) else 0)
df['crew_size'] = df['crew'].apply(lambda x: len(x) if isinstance(x, list) else 0)
df['total_team_size'] = df['cast_size'] + df['crew_size']

# (6) 是否有女性演员（简单判断）
df['has_female_actor'] = df['cast'].apply(
    lambda x: any(actor.get('gender') == 1 for actor in x if isinstance(actor, dict))
    if isinstance(x, list) else False
)


# ========================
# 4. 处理缺失值与异常
# ========================

# 填充导演为空的情况
df['director'] = df['director'].fillna('Unknown')
df['cinematographer'] = df['cinematographer'].fillna('Unknown')

# 可选：删除无效行（cast 和 crew 都为空）
df_clean = df[(df['cast_size'] > 0) | (df['crew_size'] > 0)].copy()

print(f"🧹 清洗后有效数据行数: {len(df_clean)} (原 {len(df)})")


# ========================
# 5. 展示处理结果
# ========================

print("\n✨ 处理后的关键字段预览:")
preview_cols = ['title', 'director', 'top_3_actors', 'writers', 'cast_size', 'crew_size', 'total_team_size', 'has_female_actor']
print(df_clean[preview_cols].head(8).to_string(index=False), "\n")


# ========================
# 6. 基础统计分析
# ========================

print("📊 基础统计分析:")

# 导演出现频率 Top 10
director_top10 = df_clean['director'].value_counts().head(10)
print(f"\n🎬 导演作品数量 Top 10:\n{director_top10}")

# 团队规模分布
print(f"\n👥 团队总人数统计:")
print(f"  平均: {df_clean['total_team_size'].mean():.1f}")
print(f"  最大: {df_clean['total_team_size'].max()}")
print(f"  最小: {df_clean['total_team_size'].min()}")

# 女性演员参与率
female_ratio = df_clean['has_female_actor'].mean()
print(f"\n👩 女性演员参与率: {female_ratio:.1%}")


# ========================
# 7. 保存清洗后数据
# ========================

output_file = 'tmdb_credits_cleaned.csv'
df_clean.to_csv(output_file, index=False, encoding='utf-8-sig')  # utf-8-sig 支持 Excel 打开中文

print(f"\n🎉 数据预处理完成！")
print(f"💾 已保存为: '{output_file}'")
print(f"📦 包含字段: {list(df_clean.columns)}")
